<?php
// composer下载方式
// 先使用composer命令下载：
// composer require owner888/phpspider
// 引入加载器
//require './vendor/autoload.php';

// GitHub下载方式
require_once __DIR__ . '/../autoloader.php';
    use phpspider\core\db;
    use phpspider\core\phpspider;
    use phpspider\core\requests;
    
    /* Do NOT delete this comment */
/* 不要删除这段注释 */
/**
 * 实例化阿里云oos
 * @return object 实例化得到的对象
 * @throws \OSS\Core\OssException
 */
function new_oss(){
    require_once __DIR__."/../../simplewind/Core/Library/Vendor/Alioss/autoload.php";
    $config=array(
        'KEY_ID'             => 'LTAItUsYcUr43aWQ', // 阿里云oss key_id
        'KEY_SECRET'         => 'dseeyXS9xmDqIzehY2YI91JosbdGQe', // 阿里云oss key_secret
        'END_POINT'          => 'http://oss-cn-beijing.aliyuncs.com', // 阿里云oss endpoint 华北2
        'BUCKET'             => 'dbcc0'  // bucken 名称
    );
    $oss=new \OSS\OssClient($config['KEY_ID'],$config['KEY_SECRET'],$config['END_POINT']);
    return $oss;
}

/**
 * 上传文件到oss并删除本地文件
 * @param  string $path 文件路径
 * @return bool 是否上传
 * @throws \OSS\Core\OssException
 */
function oss_upload($path){
    // 获取配置项
    $bucket='dbcc0';
    // 先统一去除左侧的.或者/ 再添加./
    $oss_path=ltrim($path,'./');
    $path='./'.$path;
    if (file_exists($path)) {
        // 实例化oss类
        $oss=new_oss();
        // 上传到oss
        $oss->uploadFile($bucket,$oss_path,$path);
        // 如需上传到oss后 自动删除本地的文件 则删除下面的注释
        unlink($path);
        return true;
    }
    return false;
}
$configs = array(
    'name' => 'pixabay',
    'log_show' => true,
    'tasknum' => 4,
    'save_running_state' => true,
    'domains' => array(
        'pixabay.com',
    ),
    'scan_urls' => array(
        'https://pixabay.com/'
    ),
    'list_url_regexes' => array(
        "https://pixabay.com/zh/editors_choice/",
        "https://pixabay.com/zh/editors_choice/?media_type=photo&pagi=\d+",
        "https://pixabay.com/zh/editors_choice/?media_type=illustration&pagi=\d+",
        "https://pixabay.com/zh/editors_choice/?media_type=vector&pagi=\d+",
        "https://pixabay.com/zh/photos/?&pagi=\d+"
        //"https://pixabay.com/zh/editors_choice/?media_type=video&pagi=\d+"
    ),
    'content_url_regexes' => array(
        "https://pixabay.com/zh/image-\d+",
        //"https://pixabay.com/zh/videos/none-\d+",
    ),
    'max_try' => 5,
    //'proxies' => array(
        //'http://H784U84R444YABQD:57A8B0B743F9B4D2@proxy.abuyun.com:9010'
    //),
    //'export' => array(
        //'type' => 'csv',
        //'file' => '../data/qiushibaike.csv',
    //),
    //'export' => array(
        //'type'  => 'sql',
        //'file'  => '../data/qiushibaike.sql',
        //'table' => 'content',
    //),
    //链接数据库表
    'export' => array(
        'type' => 'db',
        'table' => 'cco_images',
    ),
    //链接数据库
    'db_config' => array(
        'host'  => '127.0.0.1',
        'port'  => 3306,
        'user'  => 'root',
        'pass'  => '15726818351',
        'name'  => 'ios',
    ),
    //redis
    'queue_config' => array(
        'host'      => '127.0.0.1',
        'port'      => 6379,
        'pass'      => '',
        'db'        => 5,
        'prefix'    => 'pixabay1',
        'timeout'   => 30,
    ),
    //过滤要执行的数据
    'fields' => array(
        array(
            'name' => "title",
            'selector' => "//*[@id='media_container']/img/@alt",
            'required' => true,
        ),
        array(
            'name' => "pic",
            'selector' => "//*[@id='media_container']/img/@srcset",
            'required' => true,
        ),
        array(
            'name' => "label",
            'selector' => "//*[@id='media_container']/img/@alt",
            'required' => true,
        ),
        array(
            'name' => "protocol",
            'selector' => "//*[@id='media_show']/div[2]/div[5]/a",
            'required' => true,
        ),
        array(
            'name' => "create_time",
            'selector' => "//*[@id='media_container']/img/@src",
            'required' => true,
        ),
        array(
            'name' => "width",
            'selector' => "//*[@id='media_container']/img/@srcset",
            'required' => true,
        ),
        array(
            'name' => "height",
            'selector' => "//*[@id='media_container']/img/@srcset",
            'required' => true,
        ),
        array(
            'name' => "type",
            'selector' => "//*[@id='media_container']/img/@srcset",
            'required' => true,
        ),
        array(
            'name' => "username",
            'selector' => "//*[@id='media_container']/img/@src",
            'required' => true,
        )
    ),
);


$spider = new phpspider($configs);
$spider->on_start = function($phpspider)
{
    $db_config = $phpspider->get_config("db_config");
    //print_r($db_config);
    //exit;
    // 数据库连接
    db::set_connect('default', $db_config);
    db::init_mysql();
    $login_url="https://pixabay.com/zh/accounts/login/?next=https://pixabay.com";
    $params = array(
        "username" => "w_1121562734",
        "password" => "wjb64800891",
    );
    // 发送登录请求
    requests::post($login_url, $params);
    // 登录成功后本框架会把Cookie保存到www.waduanzi.com域名下，我们可以看看是否是已经收集到Cookie了
    $cookies = requests::get_cookies("pixabay.com");
     //requests对象自动收集Cookie，访问这个域名下的URL会自动带上
    //print_r($cookies);  // 可以看到已经输出Cookie数组结构
    requests::set_useragent(array(
        "Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/",
        "Opera/9.80 (Android 3.2.1; Linux; Opera Tablet/ADR-1109081720; U; ja) Presto/2.8.149 Version/11.10",
        "Mozilla/5.0 (Android; Linux armv7l; rv:9.0) Gecko/20111216 Firefox/9.0 Fennec/9.0",
    ));
    
};

$spider->on_extract_field = function($fieldname, $data, $page) 
{
    if ($fieldname == 'pic')
    {
        //https://pixabay.com/zh/photos/download/bird-2823767.jpg?atttachmen
        //$url="https://pixabay.com/zh/photos/download/".$data;
        $data=explode(',', $data)[1];
        $url=explode(" ", trim($data))[0];
        //返回文件的基本信息
        $pathinfo = pathinfo($url);
        $fileext =  $pathinfo['extension'];
        //if (strtolower($fileext) == 'jpeg')
        //{
        //    $fileext = 'jpg';
        //}
        // 以纳秒为单位生成随机数
        $filename = uniqid().".".$fileext;
        $date=date("Ymd");
        $dirs=dirname(__FILE__)."/../../uploads/{$date}";
        // 在data目录下生成图片
        $filepath =  $dirs."/".$filename;
        if(!is_dir($dirs)){
            mkdir($dirs, 0777, true);
        }
        // 用系统自带的下载器wget下载
        exec("wget -q {$url} -O {$filepath}");
        $img_path = "uploads/{$date}/{$filename}";
        oss_upload($img_path);
        // 替换成真是图片url
        $data = $img_path;
    }
    elseif ($fieldname == 'create_time')
    {
        // 用当前采集时间戳作为发布时间
        if($data){
            $len=strpos($data, '20');
            $data=substr($data,$len,16);
            str_replace('/',"-", $data);
            $ye=substr($data,$len,10);
            $h=substr($data,$len+11,5);
            $ye=str_replace('/',"-", $ye);
            $h=str_replace('/',":", $h);
            $data=$ye." ".$h;
            $data=strtotime($data);
        }else{
            $data = time();
        }
    
    }elseif ($fieldname == 'protocol'){// 把当前内容页URL替换上面的field
        $str=strpos($data,'CC0');
        if($str !== false){
            $data='CC0';
        }else{
            return false;
        }
    }
    elseif($fieldname == 'width'){
        $data=explode(',', $data)[1];
        $url=explode(" ", trim($data))[0];
        $pathinfo = getimagesize($url);
        $data = $pathinfo[0];
    }
    elseif($fieldname == 'height'){
        $data=explode(',', $data)[1];
        $url=explode(" ", trim($data))[0];
        $pathinfo = getimagesize($url);
        $data =  $pathinfo[1];
    }elseif($fieldname == 'type'){
        $data=explode(',', $data)[1];
        $url=explode(" ", trim($data))[0];
        $pathinfo = pathinfo($url);
        $data =  $pathinfo['extension'];
    }elseif($fieldname == 'username'){
        $data='pixabay';
    }
    return $data;
};
    
$spider->on_extract_page = function($page,$data){
    $sql = "Select Count(*) As `count` From `cco_images` Where `title`='{$data['title']}'";
    $row = db::get_one($sql);
    //if(!$row['count']){
    //    $str=strpos($data['protocol'],'CC0');
    //    if($str){
    //        $data['protocol']='CC0';
    //        db::insert("cco_images", $data);
    //
    //    }
    //}
    if(!$row['count']){
        $str=strpos($data['protocol'],'CC0');
        if($str !== false){
            $data['protocol']='CC0';
            return $data;
        }else{
            return false;
        }
    }else{
        return false;
    }
};
$spider->start();


